{"values":{"name":"Nuotrauku importas","code":"1230","description":null,"triggerTypeId":0,"placeId":"company-form","scriptContent":"const MAX_PARALLEL = 15;\r\nawait logMessage(\"Pradėtas duomenų apdorojimas\");\r\nconst PARAMETER = await loadApiParameters(\"PRODUCTS-UPDATE\");\r\n\r\nawait main();\r\n\r\noutput = {\r\n    message: \"Importas pabaigtas\",\r\n}\r\nreturn;\r\n\r\n\r\nasync function main() {\r\n    try {\r\n        let csvData = await getCsvData();\r\n        //csvData = csvData.slice(0,MAX_PARALLEL*3)\r\n        let processed = true;\r\n        let needValidation = true;\r\n        while (processed) {\r\n            let currentPageNo = await getLastProcessedPageNo();\r\n            if (currentPageNo == null) {\r\n                currentPageNo = 0;\r\n            } else {\r\n                currentPageNo = currentPageNo + 1;\r\n            }\r\n            await logMessage(`Process PageNo: ${currentPageNo} started`);\r\n            const pageData = getPageData(currentPageNo, csvData);\r\n            if (pageData != null && pageData?.length > 0) {\r\n                processed = await processPage(pageData, needValidation); // viduje paralelus procesas\r\n                await saveLastProcessedPageNo(currentPageNo);\r\n                needValidation = false;\r\n            } else {\r\n                processed = false;\r\n            }\r\n            await logMessage(`Process Page: ${currentPageNo} finished`);\r\n        }\r\n    } catch(error) {\r\n        await logError(\"got error:\", error);\r\n    }\r\n}\r\n\r\n\r\nasync function getLastProcessedPageNo() {\r\n    if (!PARAMETER) {\r\n        throw new Error(`Neaprašytas PRODUCTS-UPDATE parametras`);\r\n    }\r\n    const lastPageNo = PARAMETER?.values.lastPageNo\r\n    return lastPageNo;\r\n}\r\n\r\n\r\nasync function saveLastProcessedPageNo(pageNo) {\r\n    PARAMETER.values.lastPageNo = pageNo;\r\n    let updatedParamter = await mutate(`updatePartialParameterValue`, {parameterValueId: PARAMETER.id, value: JSON.stringify(PARAMETER.values)})\r\n    log.trace(\"updatedParamter: \"+ JSON.stringify(updatedParamter, null, 2));\r\n}\r\n\r\n\r\nasync function getCsvData() {\r\n    const fileId = '57b92353-1d72-4fea-85c0-e6a76ca24327';\r\n    const files = await rql(`@ALL SELECT * FROM files WHERE id = ${fileId} `);    \r\n    const fileUrl = files[0].internalLink;\r\n    const fileBinary = await fileService.getFileBinaryByLink(fileUrl);    \r\n    const fileContent = fileBinary.toString('utf-8');\r\n    const csvDataAsJson = dataParser.csvToJson(fileContent,\r\n            {\r\n                valuesSeparator: \"\\t\",   // Default value: ;\r\n                rowsSeparator: \"\\n\",     // Default value: \\n\r\n                firstRowHeaders: true,  // Default value: true\r\n            });\r\n    return csvDataAsJson;\r\n}\r\n\r\n\r\nfunction getPageData(currentPage, csvData) {\r\n    const rowsPerPage = MAX_PARALLEL;\r\n    let rowFrom = currentPage * rowsPerPage; // 0*30=0,  1*30=30, 2*30=60 \r\n    let rowTo = rowFrom + rowsPerPage;       //      30,      60,      90\r\n    const pageData = csvData.slice(rowFrom, rowTo);\r\n    return pageData;\r\n}\r\n\r\n\r\nasync function processPage(pageData, needValidation) {\r\n    try {\r\n        const results = [];\r\n        const executing = new Set();\r\n        for(rowData of pageData) {\r\n            const task = processRow(rowData, needValidation);\r\n            results.push(task);\r\n            executing.add(task);\r\n            task.finally(() => executing.delete(task));\r\n            if (executing.size >= MAX_PARALLEL) {\r\n                // if (executing.size >= maxConcurrency) {\r\n                await Promise.race(executing);\r\n            }\r\n        }\r\n        await Promise.all(executing);\r\n        Promise.all(results);\r\n        return true;\r\n    } catch(error) {\r\n        await logError(\"processPage got error:\", error);\r\n        throw error;\r\n    }\r\n}\r\n\r\n\r\nasync function processRow(rowData, needValidation) {\r\n    try {\r\n        await log.trace(\"processRow: \"+ JSON.stringify(rowData, null, 2));\r\n        let productFilesData = await processProductData(rowData);\r\n        \r\n        const productId = productFilesData[0].productId;\r\n        log.trace(`processRow: ${productId} started`);\r\n        if (productFilesData == null || productFilesData?.length < 1) return;\r\n        if (needValidation) {\r\n            productFilesData = await getNotExistFilesInProduct(productId, productFilesData);\r\n         }\r\n        for (const productFileData of productFilesData) {\r\n            await uploadProductFile(productFileData);\r\n        }\r\n        log.trace(`processRow: ${productId} finished`);\r\n    } catch(error) {\r\n        await logError(\"processRow got error:\", error);\r\n        throw error;\r\n    }\r\n}\r\n\r\n\r\nasync function getNotExistFilesInProduct(productId, productFilesData) {\r\n    const files = await rql(`@ALL SELECT name FROM entityAttachments WITH entityTypeId = 68edbe9c-159a-4f72-a45b-08b5a7c903cc refId = ${productId}`)\r\n    const result = productFilesData?.filter(e => !files.some(f => f?.name === e?.name)) ?? []\r\n    return result;\r\n}\r\n\r\n\r\nasync function uploadProductFile(item) {\r\n    if (!item?.productId || !item?.url) {\r\n        return\r\n    }\r\n    try {\r\n        const fileBase64 = await fetchImageAsBase64(item.url);\r\n        await uploadFile(fileBase64, item.fileName, item.productId, item.tags);\r\n        await log.trace(`Uploaded file: ${item.url}`);\r\n    } catch(error) {\r\n        await logError(`uploadProductFile ${item.url} got error:`, error);\r\n        throw error;\r\n    }\r\n}\r\n\r\n/************************************/\r\n/************************************/\r\n\r\n\r\nasync function processProductData(productItem) {\r\n    const processedData = []; // Array to hold the structured product data\r\n\r\n    const productId = await getProductIdByCode(productItem[\"##refId##\"]);\r\n    if (productId == null) return;\r\n\r\n    const mainImageUrl = productItem[\"Varianto nuotraukos\"]; // The main image URL\r\n    let isMain = true;\r\n\r\n    if (mainImageUrl?.length > 0) {\r\n        isMain = false; // The first image is no longer \"main\" if a main image is specified\r\n        const mainImageData = {\r\n            fileName: getFileNameFromUrl(mainImageUrl),\r\n            productId,\r\n            url: mainImageUrl,\r\n            tags: [\"main\"], // Tag for the main image\r\n        };\r\n        processedData.push(mainImageData);\r\n    }\r\n\r\n    const additionalImageUrls = productItem[\"Prekes nuotraukos\"].split(\";\"); // Split additional image URLs\r\n\r\n    for (const imageUrl of additionalImageUrls) {\r\n        const imageData = {\r\n            fileName: getFileNameFromUrl(imageUrl),\r\n            productId,\r\n            url: imageUrl,\r\n            tags: isMain ? [\"main\"] : [], // Assign \"main\" tag only to the first additional image\r\n        };\r\n        isMain = false; // Reset the flag after the first image\r\n        processedData.push(imageData);\r\n    }\r\n\r\n    return processedData;\r\n}\r\n\r\n\r\nasync function uploadFile(fileContent, fileName, productId, tags) {\r\n    await fileService.uploadFile(\r\n        fileContent,\r\n        fileName,\r\n        {\r\n            fileType: \"IMAGE\",\r\n            tags: tags,\r\n            association: {\r\n                companyId: companyId,\r\n                entityTypeId: \"68edbe9c-159a-4f72-a45b-08b5a7c903cc\",\r\n                refId: productId,\r\n                refName: fileName\r\n            },\r\n            isTemporary: false,\r\n            generatePublicLink: false,\r\n            encoding: \"utf8\",\r\n        }\r\n    );\r\n}   \r\n\r\n\r\nasync function fetchImageAsBase64(url) {\r\n    try {\r\n        const response = await fetch(url);\r\n        if (!response.ok) {\r\n            throw new Error(`HTTP error! status: ${response.status}`);\r\n        }\r\n        const blob = await response.buffer();\r\n        return blob\r\n    } catch (error) {\r\n        throw error;\r\n    }\r\n}\r\n\r\n\r\nasync function getProductIdByCode (code) {\r\n    try {\r\n        const result = await rql(`@ALL SELECT id, code FROM products WHERE code = '${code}'`)\r\n        return result[0]?.id;\r\n    } catch (error) {\r\n        logerror('Exception getProductIdByCode:', error);\r\n    } \r\n}\r\n\r\n\r\nfunction getFileNameFromUrl(url) {\r\n    // Pašalinti URL protokolą ir domeną (https://www.voniaplius.lt)\r\n    const path = url.split('//')[1].split('/').slice(1).join('/');\r\n    // Suskaidyti kelią į segmentus pagal '/'\r\n    const segments = path.split('/');\r\n    // Paskutinė dalis bus failo pavadinimas\r\n    const fileName = segments.pop();\r\n    return fileName ;\r\n}\r\n\r\n\r\nasync function logError(message, error) {\r\n    const errorMessage = error.message;\r\n    await log.error(message + ' ' + errorMessage);\r\n}\r\n","paramsFormEnabled":false,"paramsFormSchema":null,"exampleData":"","activeFrom":null,"activeTo":null,"active":true,"appDefinitionId":"ed57460c-6450-45b2-805f-4a9a072aef61"},"additionalPlaces":[]}